<!DOCTYPE html>
<html lang=zh>
<head>
  <meta charset="utf-8">
  
  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, minimum-scale=1, user-scalable=no, minimal-ui">
  <meta name="renderer" content="webkit">
  <meta http-equiv="Cache-Control" content="no-transform" />
  <meta http-equiv="Cache-Control" content="no-siteapp" />
  <meta name="apple-mobile-web-app-capable" content="yes">
  <meta name="apple-mobile-web-app-status-bar-style" content="black">
  <meta name="format-detection" content="telephone=no,email=no,adress=no">
  <!-- Color theme for statusbar -->
  <meta name="theme-color" content="#000000" />
  <!-- 强制页面在当前窗口以独立页面显示,防止别人在框架里调用页面 -->
  <meta http-equiv="window-target" content="_top" />
  
  
  <title>如何使用huggingface的trainer训练模型？ | chadqiu&#39;s blog</title>
  <meta name="description" content="huggingface上又很多开源模型，可以直接开箱即用，一个简单的模型使用实例如下： 1234567from transformers import BertTokenizer, BertModeltokenizer &#x3D; BertTokenizer.from_pretrained(&amp;#x27;uer&#x2F;chinese_roberta_L-8_H-512&amp;#x27;)model &#x3D; BertMode">
<meta property="og:type" content="article">
<meta property="og:title" content="如何使用huggingface的trainer训练模型？">
<meta property="og:url" content="http://chadqiu.github.io/fcb5896c5f58.html">
<meta property="og:site_name" content="chadqiu">
<meta property="og:description" content="huggingface上又很多开源模型，可以直接开箱即用，一个简单的模型使用实例如下： 1234567from transformers import BertTokenizer, BertModeltokenizer &#x3D; BertTokenizer.from_pretrained(&amp;#x27;uer&#x2F;chinese_roberta_L-8_H-512&amp;#x27;)model &#x3D; BertMode">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="http://chadqiu.github.io/images/discuss_data.png">
<meta property="og:image" content="http://chadqiu.github.io/images/newcoder_data.png">
<meta property="article:published_time" content="2023-02-03T13:34:25.000Z">
<meta property="article:modified_time" content="2024-11-26T16:26:22.004Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="huggingface">
<meta property="article:tag" content="python">
<meta property="article:tag" content="NLP">
<meta property="article:tag" content="train">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://chadqiu.github.io/images/discuss_data.png">
  <!-- Canonical links -->
  <link rel="canonical" href="http://chadqiu.github.io/fcb5896c5f58.html">
  
    <link rel="alternate" href="/atom.xml" title="chadqiu" type="application/atom+xml">
  
  
    <link rel="icon" href="images/avatar.jpg" type="image/x-icon">
  
  
<link rel="stylesheet" href="/css/style.css">

  
  
  
  
<meta name="generator" content="Hexo 6.3.0"></head>


<body class="main-center theme-green" itemscope itemtype="http://schema.org/WebPage">
  <header class="header" itemscope itemtype="http://schema.org/WPHeader">
  <div class="slimContent">
    <div class="navbar-header">
      
      
      <div class="profile-block text-center">
        <a id="avatar" href="https://github.com/chadqiu" target="_blank">
          <img class="img-circle img-rotate" src="/images/logo.jpg" width="200" height="200">
        </a>
        <h2 id="name" class="hidden-xs hidden-sm">chadqiu</h2>
        <h3 id="title" class="hidden-xs hidden-sm hidden-md">Developer &amp; Researcher</h3>
        <small id="location" class="text-muted hidden-xs hidden-sm"><i class="icon icon-map-marker"></i> Shanghai, China</small>
      </div>
      
      <div class="search" id="search-form-wrap">

    <form class="search-form sidebar-form">
        <div class="input-group">
            <input type="text" class="search-form-input form-control" placeholder="搜索" />
            <span class="input-group-btn">
                <button type="submit" class="search-form-submit btn btn-flat" onclick="return false;"><i class="icon icon-search"></i></button>
            </span>
        </div>
    </form>
    <div class="ins-search">
  <div class="ins-search-mask"></div>
  <div class="ins-search-container">
    <div class="ins-input-wrapper">
      <input type="text" class="ins-search-input" placeholder="想要查找什么..." x-webkit-speech />
      <button type="button" class="close ins-close ins-selectable" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
    </div>
    <div class="ins-section-wrapper">
      <div class="ins-section-container"></div>
    </div>
  </div>
</div>


</div>
      <button class="navbar-toggle collapsed" type="button" data-toggle="collapse" data-target="#main-navbar" aria-controls="main-navbar" aria-expanded="false">
        <span class="sr-only">Toggle navigation</span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
    </div>
    <nav id="main-navbar" class="collapse navbar-collapse" itemscope itemtype="http://schema.org/SiteNavigationElement" role="navigation">
      <ul class="nav navbar-nav main-nav ">
        
        
        <li class="menu-item menu-item-home">
          <a href="/.">
            
            <i class="icon icon-home-fill"></i>
            
            <span class="menu-title">首页</span>
          </a>
        </li>
        
        
        <li class="menu-item menu-item-archives">
          <a href="/archives">
            
            <i class="icon icon-archives-fill"></i>
            
            <span class="menu-title">归档</span>
          </a>
        </li>
        
        
        <li class="menu-item menu-item-categories">
          <a href="/categories">
            
            <i class="icon icon-folder"></i>
            
            <span class="menu-title">分类</span>
          </a>
        </li>
        
        
        <li class="menu-item menu-item-tags">
          <a href="/tags">
            
            <i class="icon icon-tags"></i>
            
            <span class="menu-title">标签</span>
          </a>
        </li>
        
        
        <li class="menu-item menu-item-repository">
          <a href="/repository">
            
            <i class="icon icon-project"></i>
            
            <span class="menu-title">项目</span>
          </a>
        </li>
        
        
        <li class="menu-item menu-item-links">
          <a href="/links">
            
            <i class="icon icon-friendship"></i>
            
            <span class="menu-title">友链</span>
          </a>
        </li>
        
        
        <li class="menu-item menu-item-about">
          <a href="/about">
            
            <i class="icon icon-cup-fill"></i>
            
            <span class="menu-title">关于</span>
          </a>
        </li>
        
      </ul>
      
	
    <ul class="social-links">
    	
        <li><a href="https://github.com/chadqiu" target="_blank" title="Github" data-toggle=tooltip data-placement=top><i class="icon icon-github"></i></a></li>
        
        <li><a href="/atom.xml" target="_blank" title="Rss" data-toggle=tooltip data-placement=top><i class="icon icon-rss"></i></a></li>
        
    </ul>

    </nav>
  </div>
</header>

  
    <aside class="sidebar" itemscope itemtype="http://schema.org/WPSideBar">
  <div class="slimContent">
    
      <div class="widget">
    <h3 class="widget-title">公告</h3>
    <div class="widget-body">
        <div id="board">
            <div class="content">
                <p>欢迎交流与分享经验!</p>
            </div>
        </div>
    </div>
</div>

    
      
  <div class="widget">
    <h3 class="widget-title">分类</h3>
    <div class="widget-body">
      <ul class="category-list"><li class="category-list-item"><a class="category-list-link" href="/categories/AI/">AI</a><span class="category-list-count">3</span></li><li class="category-list-item"><a class="category-list-link" href="/categories/tools-%E7%88%AC%E8%99%AB/">tools - 爬虫</a><span class="category-list-count">3</span></li><li class="category-list-item"><a class="category-list-link" href="/categories/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80-%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84%E4%B8%8E%E7%AE%97%E6%B3%95/">计算机基础 - 数据结构与算法</a><span class="category-list-count">1</span></li></ul>
    </div>
  </div>


    
      
  <div class="widget">
    <h3 class="widget-title">标签</h3>
    <div class="widget-body">
      <ul class="tag-list" itemprop="keywords"><li class="tag-list-item"><a class="tag-list-link" href="/tags/AI/" rel="tag">AI</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Java/" rel="tag">Java</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/NLP/" rel="tag">NLP</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/dataset/" rel="tag">dataset</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/huggingface/" rel="tag">huggingface</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/python/" rel="tag">python</a><span class="tag-list-count">6</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/train/" rel="tag">train</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/%E4%BA%8C%E5%88%86%E6%90%9C%E7%B4%A2/" rel="tag">二分搜索</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/%E7%88%AC%E8%99%AB/" rel="tag">爬虫</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/%E7%89%9B%E5%AE%A2%E7%BD%91/" rel="tag">牛客网</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/%E7%AE%97%E6%B3%95/" rel="tag">算法</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/%E8%B0%B7%E6%AD%8C%E5%AD%A6%E6%9C%AF/" rel="tag">谷歌学术</a><span class="tag-list-count">1</span></li></ul>
    </div>
  </div>


    
      
  <div class="widget">
    <h3 class="widget-title">标签云</h3>
    <div class="widget-body tagcloud">
      <a href="/tags/AI/" style="font-size: 13.33px;">AI</a> <a href="/tags/Java/" style="font-size: 13px;">Java</a> <a href="/tags/NLP/" style="font-size: 13.33px;">NLP</a> <a href="/tags/dataset/" style="font-size: 13px;">dataset</a> <a href="/tags/huggingface/" style="font-size: 13.67px;">huggingface</a> <a href="/tags/python/" style="font-size: 14px;">python</a> <a href="/tags/train/" style="font-size: 13px;">train</a> <a href="/tags/%E4%BA%8C%E5%88%86%E6%90%9C%E7%B4%A2/" style="font-size: 13px;">二分搜索</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 13.67px;">爬虫</a> <a href="/tags/%E7%89%9B%E5%AE%A2%E7%BD%91/" style="font-size: 13px;">牛客网</a> <a href="/tags/%E7%AE%97%E6%B3%95/" style="font-size: 13px;">算法</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E5%AD%A6%E6%9C%AF/" style="font-size: 13px;">谷歌学术</a>
    </div>
  </div>

    
      
  <div class="widget">
    <h3 class="widget-title">归档</h3>
    <div class="widget-body">
      <ul class="archive-list"><li class="archive-list-item"><a class="archive-list-link" href="/archives/2025/03/">三月 2025</a><span class="archive-list-count">1</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2025/02/">二月 2025</a><span class="archive-list-count">2</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2025/01/">一月 2025</a><span class="archive-list-count">5</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2024/12/">十二月 2024</a><span class="archive-list-count">5</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2024/11/">十一月 2024</a><span class="archive-list-count">1</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2023/02/">二月 2023</a><span class="archive-list-count">3</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2023/01/">一月 2023</a><span class="archive-list-count">5</span></li></ul>
    </div>
  </div>


    
      
  <div class="widget">
    <h3 class="widget-title">最新文章</h3>
    <div class="widget-body">
      <ul class="recent-post-list list-unstyled no-thumbnail">
        
          <li>
            
            <div class="item-inner">
              <p class="item-category">
                
              </p>
              <p class="item-title">
                <a href="/853c4069a45a.html" class="title">Z哥25-03-02-现在到底是个什么情况？</a>
              </p>
              <p class="item-date">
                <time datetime="2025-03-03T00:40:27.000Z" itemprop="datePublished">2025-03-03</time>
              </p>
            </div>
          </li>
          
          <li>
            
            <div class="item-inner">
              <p class="item-category">
                
              </p>
              <p class="item-title">
                <a href="/44a02643ff6a.html" class="title">Z哥25-02-16直播</a>
              </p>
              <p class="item-date">
                <time datetime="2025-02-17T01:05:09.000Z" itemprop="datePublished">2025-02-17</time>
              </p>
            </div>
          </li>
          
          <li>
            
            <div class="item-inner">
              <p class="item-category">
                
              </p>
              <p class="item-title">
                <a href="/d8613ad6db3e.html" class="title">Z哥25-02-09直播</a>
              </p>
              <p class="item-date">
                <time datetime="2025-02-10T00:59:45.000Z" itemprop="datePublished">2025-02-10</time>
              </p>
            </div>
          </li>
          
          <li>
            
            <div class="item-inner">
              <p class="item-category">
                
              </p>
              <p class="item-title">
                <a href="/814464067ab6.html" class="title">Z哥25-01-27直播</a>
              </p>
              <p class="item-date">
                <time datetime="2025-01-28T14:24:20.000Z" itemprop="datePublished">2025-01-28</time>
              </p>
            </div>
          </li>
          
          <li>
            
            <div class="item-inner">
              <p class="item-category">
                
              </p>
              <p class="item-title">
                <a href="/49dc4032c5df.html" class="title">Z哥25-01-24直播</a>
              </p>
              <p class="item-date">
                <time datetime="2025-01-25T12:05:29.000Z" itemprop="datePublished">2025-01-25</time>
              </p>
            </div>
          </li>
          
      </ul>
    </div>
  </div>
  

    
  </div>
</aside>

  
  
  <aside class="sidebar sidebar-toc collapse   in  " id="collapseToc" itemscope itemtype="http://schema.org/WPSideBar">
  <div class="slimContent">
    <nav id="toc" class="article-toc">
      <h3 class="toc-title">文章目录</h3>
      <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%AE%AD%E7%BB%83%E4%B8%80%E4%B8%AANLU%E6%A8%A1%E5%9E%8B"><span class="toc-number">1.</span> <span class="toc-text">训练一个NLU模型</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9E%84%E5%BB%BA%E6%95%B0%E6%8D%AE%E9%9B%86"><span class="toc-number">1.1.</span> <span class="toc-text">构建数据集</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%9A%E4%B9%89%E8%AF%84%E4%BB%B7%E6%8C%87%E6%A0%87%E5%87%BD%E6%95%B0"><span class="toc-number">1.2.</span> <span class="toc-text">定义评价指标函数</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%8C%87%E5%AE%9A%E6%A8%A1%E5%9E%8B%E7%9A%84%E8%AE%AD%E7%BB%83%E5%8F%82%E6%95%B0"><span class="toc-number">1.3.</span> <span class="toc-text">指定模型的训练参数</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%9A%E4%B9%89trainer%E5%B9%B6%E8%BF%9B%E8%A1%8C%E8%AE%AD%E7%BB%83"><span class="toc-number">1.4.</span> <span class="toc-text">定义trainer并进行训练</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%B5%8B%E8%AF%95%E9%A2%84%E6%B5%8B"><span class="toc-number">1.5.</span> <span class="toc-text">测试预测</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BB%A3%E7%A0%81%E6%95%B4%E5%90%88"><span class="toc-number">1.6.</span> <span class="toc-text">代码整合</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A8%A1%E5%9E%8B%E6%8E%A8%E7%90%86%E9%A2%84%E6%B5%8B"><span class="toc-number">1.7.</span> <span class="toc-text">模型推理预测</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%AE%AD%E7%BB%83seq2seq%E7%94%9F%E6%88%90%E5%BC%8F%E6%A8%A1%E5%9E%8BT5"><span class="toc-number">2.</span> <span class="toc-text">训练seq2seq生成式模型T5</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9E%84%E5%BB%BA%E6%95%B0%E6%8D%AE%E9%9B%86-1"><span class="toc-number">2.1.</span> <span class="toc-text">构建数据集</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%AD%89%E4%B8%80%E8%AF%84%E4%BB%B7%E6%8C%87%E6%A0%87"><span class="toc-number">2.2.</span> <span class="toc-text">等一评价指标</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9E%84%E5%BB%BAtrainer%E8%AE%AD%E7%BB%83"><span class="toc-number">2.3.</span> <span class="toc-text">构建trainer训练</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BB%A3%E7%A0%81%E6%95%B4%E5%90%88-1"><span class="toc-number">2.4.</span> <span class="toc-text">代码整合</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A8%A1%E5%9E%8B%E6%8E%A8%E7%90%86%E9%A2%84%E6%B5%8B-1"><span class="toc-number">2.5.</span> <span class="toc-text">模型推理预测</span></a></li></ol></li></ol>
    </nav>
  </div>
</aside>

<main class="main" role="main">
  <div class="content">
  <article id="post-如何使用huggingface的trainer训练模型？" class="article article-type-post" itemscope itemtype="http://schema.org/BlogPosting">
    
    <div class="article-header">
      
        
  
    <h1 class="article-title" itemprop="name">
      如何使用huggingface的trainer训练模型？
    </h1>
  

      
      <div class="article-meta">
        <span class="article-date">
    <i class="icon icon-calendar-check"></i>
	<a href="/fcb5896c5f58.html" class="article-date">
	  <time datetime="2023-02-03T13:34:25.000Z" itemprop="datePublished">2023-02-03</time>
	</a>
</span>
        
  <span class="article-category">
    <i class="icon icon-folder"></i>
    <a class="article-category-link" href="/categories/AI/">AI</a>
  </span>

        
  <span class="article-tag">
    <i class="icon icon-tags"></i>
	<a class="article-tag-link-link" href="/tags/NLP/" rel="tag">NLP</a>, <a class="article-tag-link-link" href="/tags/huggingface/" rel="tag">huggingface</a>, <a class="article-tag-link-link" href="/tags/python/" rel="tag">python</a>, <a class="article-tag-link-link" href="/tags/train/" rel="tag">train</a>
  </span>


        

        <span class="post-comment"><i class="icon icon-comment"></i> <a href="/fcb5896c5f58.html#comments" class="article-comment-link">评论</a></span>
        
      </div>
    </div>
    <div class="article-entry marked-body" itemprop="articleBody">
      
        <p>huggingface上又很多开源模型，可以直接开箱即用，一个简单的模型使用实例如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> BertTokenizer, BertModel</span><br><span class="line">tokenizer = BertTokenizer.from_pretrained(<span class="string">&#x27;uer/chinese_roberta_L-8_H-512&#x27;</span>)</span><br><span class="line">model = BertModel.from_pretrained(<span class="string">&quot;uer/chinese_roberta_L-8_H-512&quot;</span>)</span><br><span class="line">text = <span class="string">&quot;用你喜欢的任何文本替换我。&quot;</span></span><br><span class="line">encoded_input = tokenizer(text, return_tensors=<span class="string">&#x27;pt&#x27;</span>)</span><br><span class="line">output = model(**encoded_input)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>有时候，我们需要finetune自己的模型，通常使用pytorch代码训练，写起来比较复杂，如果使用huggingface的trainer来训练就很方便了。</p>
<h2 id="训练一个NLU模型"><a href="#训练一个NLU模型" class="headerlink" title="训练一个NLU模型"></a>训练一个NLU模型</h2><p>本文将使用trainer 训练一个牛客网讨论帖文本分类模型。详细过程如下：</p>
<h3 id="构建数据集"><a href="#构建数据集" class="headerlink" title="构建数据集"></a>构建数据集</h3><p>数据集下载链接：<br><a target="_blank" rel="noopener" href="https://github.com/chadqiu/newcoder-crawler/blob/main/train.csv">train data</a><br><a target="_blank" rel="noopener" href="https://github.com/chadqiu/newcoder-crawler/blob/main/test.csv">test data</a><br>正常的训练演示用这两个数据集就够了，如果需要训练很精确的模型，可以使用伪标签大数据集<a target="_blank" rel="noopener" href="https://github.com/chadqiu/newcoder-crawler/blob/main/generated_pesudo_data.csv">generated pesudo data</a><br>数据集的结构如下：<br><img src="/images/discuss_data.png" alt="dataset"><br>每条数据包含一个文本和一个label，label为： [招聘信息、 经验贴、 求助贴] 三种类型之一。<br>我们需要加载数据集，并将文本tokenize成id，代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"><span class="keyword">from</span> datasets <span class="keyword">import</span> load_dataset</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> AutoTokenizer, AutoModelForMaskedLM, AutoModelForSequenceClassification</span><br><span class="line"></span><br><span class="line">model_name = <span class="string">&quot;bert-base-chinese&quot;</span></span><br><span class="line"></span><br><span class="line">max_input_length = <span class="number">128</span></span><br><span class="line">label2id = &#123;</span><br><span class="line">    <span class="string">&#x27;招聘信息&#x27;</span>:<span class="number">0</span>,</span><br><span class="line">    <span class="string">&#x27;经验贴&#x27;</span>:<span class="number">1</span>,</span><br><span class="line">    <span class="string">&#x27;求助贴&#x27;</span>:<span class="number">2</span></span><br><span class="line">&#125;</span><br><span class="line">id2label = &#123;v:k <span class="keyword">for</span> k,v <span class="keyword">in</span> label2id.items()&#125;</span><br><span class="line"></span><br><span class="line">tokenizer = AutoTokenizer.from_pretrained(model_name)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">preprocess_function</span>(<span class="params">examples</span>):</span><br><span class="line">    model_inputs = tokenizer(examples[<span class="string">&quot;text&quot;</span>], max_length=max_input_length, truncation=<span class="literal">True</span>)</span><br><span class="line">    labels = [label2id[x] <span class="keyword">for</span> x <span class="keyword">in</span> examples[<span class="string">&#x27;target&#x27;</span>]]</span><br><span class="line">    model_inputs[<span class="string">&quot;labels&quot;</span>] = labels</span><br><span class="line">    <span class="keyword">return</span> model_inputs</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">raw_datasets = load_dataset(<span class="string">&#x27;csv&#x27;</span>, data_files=&#123;<span class="string">&#x27;train&#x27;</span>: <span class="string">&#x27;train.csv&#x27;</span>, <span class="string">&#x27;test&#x27;</span>: <span class="string">&#x27;test.csv&#x27;</span>&#125;)</span><br><span class="line">tokenized_datasets = raw_datasets.<span class="built_in">map</span>(preprocess_function, batched=<span class="literal">True</span>, remove_columns=raw_datasets[<span class="string">&#x27;train&#x27;</span>].column_names)</span><br></pre></td></tr></table></figure>

<h3 id="定义评价指标函数"><a href="#定义评价指标函数" class="headerlink" title="定义评价指标函数"></a>定义评价指标函数</h3><p>评价指标metric用于evaluate的时候衡量模型的表现，这里使用f1 score 和 accuracy</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> sklearn.metrics <span class="keyword">import</span> f1_score, accuracy_score, classification_report</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> EvalPrediction</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">multi_label_metrics</span>(<span class="params">predictions, labels, threshold=<span class="number">0.5</span></span>):</span><br><span class="line">    probs =  np.argmax( predictions, -<span class="number">1</span>)       </span><br><span class="line">    y_true = labels</span><br><span class="line">    f1_micro_average = f1_score(y_true=y_true, y_pred=probs, average=<span class="string">&#x27;micro&#x27;</span>)</span><br><span class="line">    accuracy = accuracy_score(y_true, probs)</span><br><span class="line">    <span class="built_in">print</span>(classification_report([id2label[x] <span class="keyword">for</span> x <span class="keyword">in</span> y_true], [id2label[x] <span class="keyword">for</span> x <span class="keyword">in</span> probs]))</span><br><span class="line">    <span class="comment"># return as dictionary</span></span><br><span class="line">    metrics = &#123;<span class="string">&#x27;f1&#x27;</span>: f1_micro_average,</span><br><span class="line">               <span class="string">&#x27;accuracy&#x27;</span>: accuracy&#125;</span><br><span class="line">    <span class="keyword">return</span> metrics</span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compute_metrics</span>(<span class="params">p: EvalPrediction</span>):</span><br><span class="line">    preds = p.predictions[<span class="number">0</span>] <span class="keyword">if</span> <span class="built_in">isinstance</span>(p.predictions, <span class="built_in">tuple</span>) <span class="keyword">else</span> p.predictions</span><br><span class="line">    result = multi_label_metrics(predictions=preds, labels=p.label_ids)</span><br><span class="line">    <span class="keyword">return</span> result</span><br></pre></td></tr></table></figure>

<h3 id="指定模型的训练参数"><a href="#指定模型的训练参数" class="headerlink" title="指定模型的训练参数"></a>指定模型的训练参数</h3><p>加载模型，并构建TrainingArguments类，用于指定模型训练的各种参数<br>第一个是训练保存地址为必填项，其他都是选填项</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> TrainingArguments, Trainer</span><br><span class="line"></span><br><span class="line">batch_size = <span class="number">64</span></span><br><span class="line"></span><br><span class="line">training_args = TrainingArguments(</span><br><span class="line">    <span class="string">f&quot;/root/autodl-tmp/run&quot;</span>,</span><br><span class="line">    evaluation_strategy = <span class="string">&quot;epoch&quot;</span>,</span><br><span class="line">    save_strategy = <span class="string">&quot;epoch&quot;</span>,</span><br><span class="line">    learning_rate=<span class="number">2e-4</span>,</span><br><span class="line">    per_device_train_batch_size=batch_size,</span><br><span class="line">    per_device_eval_batch_size=batch_size,</span><br><span class="line">    <span class="comment"># gradient_accumulation_steps=2,</span></span><br><span class="line">    num_train_epochs=<span class="number">10</span>,</span><br><span class="line">    save_total_limit=<span class="number">1</span>,</span><br><span class="line">    weight_decay=<span class="number">0.01</span>,</span><br><span class="line">    load_best_model_at_end=<span class="literal">True</span>,</span><br><span class="line">    metric_for_best_model=metric_name,</span><br><span class="line">    fp16=<span class="literal">True</span>,</span><br><span class="line">)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="定义trainer并进行训练"><a href="#定义trainer并进行训练" class="headerlink" title="定义trainer并进行训练"></a>定义trainer并进行训练</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">trainer = Trainer(</span><br><span class="line">    model,</span><br><span class="line">    training_args,</span><br><span class="line">    train_dataset=tokenized_datasets[<span class="string">&quot;train&quot;</span>],</span><br><span class="line">    eval_dataset=tokenized_datasets[<span class="string">&quot;test&quot;</span>],</span><br><span class="line">    tokenizer=tokenizer,</span><br><span class="line">    compute_metrics=compute_metrics</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">trainer.train()  <span class="comment"># 开始训练</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="测试预测"><a href="#测试预测" class="headerlink" title="测试预测"></a>测试预测</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">print</span>(<span class="string">&quot;test&quot;</span>)</span><br><span class="line"><span class="built_in">print</span>(trainer.evaluate())  <span class="comment"># 测试</span></span><br><span class="line">trainer.save_model(<span class="string">&quot;bert&quot;</span>)  <span class="comment">#保存模型</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行模型预测，并将预测结果输出便于观察</span></span><br><span class="line">predictions, labels, _ = trainer.predict(tokenized_datasets[<span class="string">&quot;test&quot;</span>])</span><br><span class="line">predictions = np.argmax(predictions, axis=-<span class="number">1</span>)</span><br><span class="line"><span class="built_in">print</span>(predictions)</span><br><span class="line"><span class="built_in">print</span>(labels)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="代码整合"><a href="#代码整合" class="headerlink" title="代码整合"></a>代码整合</h3><p>将上面代码整合到一起，结果如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> datasets <span class="keyword">import</span> load_dataset</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> AutoTokenizer, AutoModelForMaskedLM, AutoModelForSequenceClassification</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> TrainingArguments, Trainer</span><br><span class="line"><span class="keyword">from</span> sklearn.metrics <span class="keyword">import</span> f1_score, roc_auc_score, accuracy_score, classification_report</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> EvalPrediction</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> evaluate</span><br><span class="line"></span><br><span class="line">metric = evaluate.load(<span class="string">&quot;seqeval&quot;</span>)</span><br><span class="line"></span><br><span class="line">model_name = <span class="string">&quot;uer/chinese_roberta_L-4_H-512&quot;</span></span><br><span class="line">tokenizer = AutoTokenizer.from_pretrained(model_name)</span><br><span class="line"></span><br><span class="line">max_input_length = <span class="number">128</span></span><br><span class="line">label2id = &#123;</span><br><span class="line">    <span class="string">&#x27;招聘信息&#x27;</span>:<span class="number">0</span>,</span><br><span class="line">    <span class="string">&#x27;经验贴&#x27;</span>:<span class="number">1</span>,</span><br><span class="line">    <span class="string">&#x27;求助贴&#x27;</span>:<span class="number">2</span></span><br><span class="line">&#125;</span><br><span class="line">id2label = &#123;v:k <span class="keyword">for</span> k,v <span class="keyword">in</span> label2id.items()&#125;</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">preprocess_function</span>(<span class="params">examples</span>):</span><br><span class="line">    model_inputs = tokenizer(examples[<span class="string">&quot;text&quot;</span>], max_length=max_input_length, truncation=<span class="literal">True</span>)</span><br><span class="line">    labels = [label2id[x] <span class="keyword">for</span> x <span class="keyword">in</span> examples[<span class="string">&#x27;target&#x27;</span>]]</span><br><span class="line">    model_inputs[<span class="string">&quot;labels&quot;</span>] = labels</span><br><span class="line">    <span class="keyword">return</span> model_inputs</span><br><span class="line"></span><br><span class="line">raw_datasets = load_dataset(<span class="string">&#x27;csv&#x27;</span>, data_files=&#123;<span class="string">&#x27;train&#x27;</span>: <span class="string">&#x27;train.csv&#x27;</span>, <span class="string">&#x27;test&#x27;</span>: <span class="string">&#x27;test.csv&#x27;</span>&#125;)</span><br><span class="line">tokenized_datasets = raw_datasets.<span class="built_in">map</span>(preprocess_function, batched=<span class="literal">True</span>, remove_columns=raw_datasets[<span class="string">&#x27;train&#x27;</span>].column_names)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">multi_label_metrics</span>(<span class="params">predictions, labels, threshold=<span class="number">0.5</span></span>):</span><br><span class="line">    probs =  np.argmax( predictions, -<span class="number">1</span>)       </span><br><span class="line">    y_true = labels</span><br><span class="line">    f1_micro_average = f1_score(y_true=y_true, y_pred=probs, average=<span class="string">&#x27;micro&#x27;</span>)</span><br><span class="line">    accuracy = accuracy_score(y_true, probs)</span><br><span class="line">    <span class="built_in">print</span>(classification_report([id2label[x] <span class="keyword">for</span> x <span class="keyword">in</span> y_true], [id2label[x] <span class="keyword">for</span> x <span class="keyword">in</span> probs]))</span><br><span class="line">    <span class="comment"># return as dictionary</span></span><br><span class="line">    metrics = &#123;<span class="string">&#x27;f1&#x27;</span>: f1_micro_average,</span><br><span class="line">               <span class="string">&#x27;accuracy&#x27;</span>: accuracy&#125;</span><br><span class="line">    <span class="keyword">return</span> metrics</span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compute_metrics</span>(<span class="params">p: EvalPrediction</span>):</span><br><span class="line">    preds = p.predictions[<span class="number">0</span>] <span class="keyword">if</span> <span class="built_in">isinstance</span>(p.predictions, <span class="built_in">tuple</span>) <span class="keyword">else</span> p.predictions</span><br><span class="line">    result = multi_label_metrics(predictions=preds, labels=p.label_ids)</span><br><span class="line">    <span class="keyword">return</span> result</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">model = AutoModelForSequenceClassification.from_pretrained(model_name, </span><br><span class="line">                                        <span class="comment"># problem_type=&quot;multi_label_classification&quot;, </span></span><br><span class="line">                                        num_labels=<span class="number">3</span>,</span><br><span class="line">                                        <span class="comment"># id2label=id2label,</span></span><br><span class="line">                                        <span class="comment"># label2id=label2id</span></span><br><span class="line">                                        )</span><br><span class="line"></span><br><span class="line">batch_size = <span class="number">64</span></span><br><span class="line">metric_name = <span class="string">&quot;f1&quot;</span></span><br><span class="line"></span><br><span class="line">training_args = TrainingArguments(</span><br><span class="line">    <span class="string">f&quot;/root/autodl-tmp/run&quot;</span>,</span><br><span class="line">    evaluation_strategy = <span class="string">&quot;epoch&quot;</span>,</span><br><span class="line">    save_strategy = <span class="string">&quot;epoch&quot;</span>,</span><br><span class="line">    learning_rate=<span class="number">2e-4</span>,</span><br><span class="line">    per_device_train_batch_size=batch_size,</span><br><span class="line">    per_device_eval_batch_size=batch_size,</span><br><span class="line">    <span class="comment"># gradient_accumulation_steps=2,</span></span><br><span class="line">    num_train_epochs=<span class="number">10</span>,</span><br><span class="line">    save_total_limit=<span class="number">1</span>,</span><br><span class="line">    weight_decay=<span class="number">0.01</span>,</span><br><span class="line">    load_best_model_at_end=<span class="literal">True</span>,</span><br><span class="line">    metric_for_best_model=metric_name,</span><br><span class="line">    fp16=<span class="literal">True</span>,</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">trainer = Trainer(</span><br><span class="line">    model,</span><br><span class="line">    training_args,</span><br><span class="line">    train_dataset=tokenized_datasets[<span class="string">&quot;train&quot;</span>],</span><br><span class="line">    eval_dataset=tokenized_datasets[<span class="string">&quot;test&quot;</span>],</span><br><span class="line">    tokenizer=tokenizer,</span><br><span class="line">    compute_metrics=compute_metrics</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">trainer.train()</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;test&quot;</span>)</span><br><span class="line"><span class="built_in">print</span>(trainer.evaluate())</span><br><span class="line">trainer.save_model(<span class="string">&quot;bert&quot;</span>)</span><br><span class="line"></span><br><span class="line">predictions, labels, _ = trainer.predict(tokenized_datasets[<span class="string">&quot;test&quot;</span>])</span><br><span class="line">predictions = np.argmax(predictions, axis=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(predictions)</span><br><span class="line"><span class="built_in">print</span>(labels)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="模型推理预测"><a href="#模型推理预测" class="headerlink" title="模型推理预测"></a>模型推理预测</h3><p>使用训练好的模型在其他数据集上推理预测，新数据集是从牛客网爬取的帖子信息,接近4万条，数据链接： <a target="_blank" rel="noopener" href="https://github.com/chadqiu/newcoder-crawler/blob/main/historical_data.xlsx">historical_data</a><br>数据截图如下：<br><img src="/images/newcoder_data.png" alt="historical_data"></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification</span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">data = pd.read_excel(<span class="string">&quot;historical_data.xlsx&quot;</span>, sheet_name=<span class="number">0</span>).fillna(<span class="string">&quot; &quot;</span>)</span><br><span class="line">data[<span class="string">&#x27;text&#x27;</span>] = data[<span class="string">&#x27;title&#x27;</span>].apply(<span class="keyword">lambda</span> x : <span class="built_in">str</span>(x) <span class="keyword">if</span> x <span class="keyword">else</span> <span class="string">&quot;&quot;</span>) + data[<span class="string">&#x27;content&#x27;</span>].apply(<span class="keyword">lambda</span> x : <span class="built_in">str</span>(x) <span class="keyword">if</span> x <span class="keyword">else</span> <span class="string">&quot;&quot;</span>)</span><br><span class="line"></span><br><span class="line">model_name = <span class="string">&quot;bert&quot;</span></span><br><span class="line">model = AutoModelForSequenceClassification.from_pretrained(model_name)</span><br><span class="line">tokenizer = AutoTokenizer.from_pretrained(model_name)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> torch.cuda.is_available():</span><br><span class="line">    device = <span class="string">&quot;cuda:0&quot;</span></span><br><span class="line">    model.half()</span><br><span class="line"><span class="keyword">else</span>:</span><br><span class="line">    device = <span class="string">&quot;cpu&quot;</span></span><br><span class="line">model = model.to(device)</span><br><span class="line"></span><br><span class="line">max_target_length = <span class="number">128</span></span><br><span class="line">label2id = &#123;</span><br><span class="line">    <span class="string">&#x27;招聘信息&#x27;</span>:<span class="number">0</span>,</span><br><span class="line">    <span class="string">&#x27;经验贴&#x27;</span>:<span class="number">1</span>,</span><br><span class="line">    <span class="string">&#x27;求助贴&#x27;</span>:<span class="number">2</span></span><br><span class="line">&#125;</span><br><span class="line">id2label = &#123;v:k <span class="keyword">for</span> k,v <span class="keyword">in</span> label2id.items()&#125;</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">get_answer</span>(<span class="params">text</span>):</span><br><span class="line">    text = [x <span class="keyword">for</span> x <span class="keyword">in</span> text]</span><br><span class="line">    inputs = tokenizer( text, return_tensors=<span class="string">&quot;pt&quot;</span>, max_length=max_target_length, padding=<span class="literal">True</span>, truncation=<span class="literal">True</span>)</span><br><span class="line">    inputs = &#123;k:v.to(device) <span class="keyword">for</span> k,v <span class="keyword">in</span> inputs.items()&#125;</span><br><span class="line">    <span class="keyword">with</span> torch.no_grad():</span><br><span class="line">        outputs = model(**inputs).logits.argmax(-<span class="number">1</span>).tolist()</span><br><span class="line">    <span class="keyword">return</span> outputs</span><br><span class="line"></span><br><span class="line"><span class="comment"># print(get_answer(data[&#x27;text&#x27;][:10]))</span></span><br><span class="line"></span><br><span class="line">pred , grod = [], []</span><br><span class="line">index, batch_size = <span class="number">0</span>, <span class="number">32</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">while</span> index &lt; <span class="built_in">len</span>(data[<span class="string">&#x27;text&#x27;</span>]):</span><br><span class="line">    pred.extend(get_answer([x <span class="keyword">for</span> x <span class="keyword">in</span> data[<span class="string">&#x27;text&#x27;</span>][index:index + batch_size]]))</span><br><span class="line">    index += batch_size</span><br><span class="line"></span><br><span class="line"><span class="comment"># print(pred)</span></span><br><span class="line"><span class="comment"># print(grod)</span></span><br><span class="line"></span><br><span class="line">pred = [id2label[x] <span class="keyword">for</span> x <span class="keyword">in</span> pred]</span><br><span class="line">data[<span class="string">&quot;target&quot;</span>] = pred</span><br><span class="line"></span><br><span class="line">writer = pd.ExcelWriter(<span class="string">&quot;generate.xlsx&quot;</span>)</span><br><span class="line">data.to_excel(writer, index=<span class="literal">False</span>, encoding=<span class="string">&#x27;utf-8&#x27;</span>, sheet_name=<span class="string">&#x27;Sheet1&#x27;</span>)</span><br><span class="line">writer.save()</span><br><span class="line">writer.close()</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="训练seq2seq生成式模型T5"><a href="#训练seq2seq生成式模型T5" class="headerlink" title="训练seq2seq生成式模型T5"></a>训练seq2seq生成式模型T5</h2><p>上面的例子是判别式模型，只用到了encoder，接下来训练一个encoder-decoder base的生成式模型T5，使用prompt用于训练，prompt方式如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="built_in">input</span>:</span><br><span class="line">请问下面文本属于哪一类帖子？</span><br><span class="line">秋招大结局（泪目了）。家人们泪目了，一波三折之后获得的小奖状，已经准备春招了，没想到被捞啦，嗐，总之是有个结果，还是很开心的[掉小珍珠了][掉小珍珠了]</span><br><span class="line">选项：招聘信息, 经验贴, 求助贴</span><br><span class="line">答案：</span><br><span class="line"></span><br><span class="line">output:</span><br><span class="line">经验贴</span><br></pre></td></tr></table></figure>

<h3 id="构建数据集-1"><a href="#构建数据集-1" class="headerlink" title="构建数据集"></a>构建数据集</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> datasets <span class="keyword">import</span> load_dataset, load_metric</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> AutoModelForSeq2SeqLM, T5Tokenizer</span><br><span class="line"></span><br><span class="line">model_name = <span class="string">&quot;ClueAI/ChatYuan-large-v1&quot;</span></span><br><span class="line">model = AutoModelForSeq2SeqLM.from_pretrained(model_name)</span><br><span class="line">tokenizer = T5Tokenizer.from_pretrained(model_name)</span><br><span class="line"></span><br><span class="line">max_input_length = <span class="number">128</span></span><br><span class="line">max_target_length = <span class="number">20</span></span><br><span class="line">prefix = <span class="string">&quot;请问下面文本属于 招聘信息、 经验贴、 求助贴 三者中的哪一类？\n&quot;</span></span><br><span class="line">suffix = <span class="string">&quot;\n选项：招聘信息, 经验贴, 求助贴\n答案：&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">preprocess_function</span>(<span class="params">examples</span>):</span><br><span class="line">    inputs = [prefix + doc + suffix <span class="keyword">for</span> doc <span class="keyword">in</span> examples[<span class="string">&quot;text&quot;</span>]]</span><br><span class="line">    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># Setup the tokenizer for targets</span></span><br><span class="line">    <span class="keyword">with</span> tokenizer.as_target_tokenizer():</span><br><span class="line">        labels = tokenizer(examples[<span class="string">&quot;target&quot;</span>], max_length=max_target_length, truncation=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">    model_inputs[<span class="string">&quot;labels&quot;</span>] = labels[<span class="string">&quot;input_ids&quot;</span>]</span><br><span class="line">    <span class="keyword">return</span> model_inputs</span><br><span class="line"></span><br><span class="line">raw_datasets = load_dataset(<span class="string">&#x27;csv&#x27;</span>, data_files=&#123;<span class="string">&#x27;train&#x27;</span>: <span class="string">&#x27;train.csv&#x27;</span>, <span class="string">&#x27;test&#x27;</span>: <span class="string">&#x27;test.csv&#x27;</span>&#125;)</span><br><span class="line">tokenized_datasets = raw_datasets.<span class="built_in">map</span>(preprocess_function, batched=<span class="literal">True</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="等一评价指标"><a href="#等一评价指标" class="headerlink" title="等一评价指标"></a>等一评价指标</h3><p>这次使用不一样的方式来构建评价指标</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> evaluate</span><br><span class="line">metric = evaluate.load(<span class="string">&quot;seqeval&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compute_metrics</span>(<span class="params">eval_pred</span>):</span><br><span class="line">    predictions, labels = eval_pred</span><br><span class="line">    decoded_preds = [tokenizer.batch_decode(predictions, skip_special_tokens=<span class="literal">True</span>)] </span><br><span class="line">    </span><br><span class="line">    <span class="comment"># Replace -100 in the labels as we can&#x27;t decode them.</span></span><br><span class="line">    labels = np.where(labels != -<span class="number">100</span>, labels, tokenizer.pad_token_id)</span><br><span class="line">    decoded_labels = [tokenizer.batch_decode(labels, skip_special_tokens=<span class="literal">True</span>)] </span><br><span class="line">    <span class="keyword">return</span> metric.compute(predictions=decoded_preds, references=decoded_labels)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="构建trainer训练"><a href="#构建trainer训练" class="headerlink" title="构建trainer训练"></a>构建trainer训练</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer</span><br><span class="line"></span><br><span class="line">batch_size = <span class="number">4</span></span><br><span class="line"></span><br><span class="line">args = Seq2SeqTrainingArguments(</span><br><span class="line">    <span class="string">f&quot;yuan-finetuned-xsum&quot;</span>,</span><br><span class="line">    evaluation_strategy = <span class="string">&quot;epoch&quot;</span>,</span><br><span class="line">    learning_rate=<span class="number">5e-5</span>,</span><br><span class="line">    per_device_train_batch_size=batch_size,</span><br><span class="line">    per_device_eval_batch_size=batch_size * <span class="number">10</span>,</span><br><span class="line">    weight_decay=<span class="number">0.01</span>,</span><br><span class="line">    save_total_limit=<span class="number">3</span>,</span><br><span class="line">    num_train_epochs=<span class="number">3</span>,</span><br><span class="line">    predict_with_generate=<span class="literal">True</span>,</span><br><span class="line">    <span class="comment"># fp16=True,</span></span><br><span class="line">    <span class="comment"># push_to_hub=True,</span></span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)</span><br><span class="line"></span><br><span class="line">trainer = Seq2SeqTrainer(</span><br><span class="line">    model,</span><br><span class="line">    args,</span><br><span class="line">    train_dataset=tokenized_datasets[<span class="string">&quot;train&quot;</span>],</span><br><span class="line">    eval_dataset=tokenized_datasets[<span class="string">&quot;test&quot;</span>],</span><br><span class="line">    data_collator=data_collator,</span><br><span class="line">    tokenizer=tokenizer,</span><br><span class="line">    compute_metrics=compute_metrics</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">trainer.train()</span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;test&quot;</span>)</span><br><span class="line"><span class="built_in">print</span>(trainer.evaluate())</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="代码整合-1"><a href="#代码整合-1" class="headerlink" title="代码整合"></a>代码整合</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> datasets <span class="keyword">import</span> load_dataset, load_metric</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer</span><br><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> AutoModelForSeq2SeqLM, T5Tokenizer</span><br><span class="line"><span class="keyword">import</span> evaluate</span><br><span class="line">metric = evaluate.load(<span class="string">&quot;seqeval&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compute_metrics</span>(<span class="params">eval_pred</span>):</span><br><span class="line">    predictions, labels = eval_pred</span><br><span class="line">    decoded_preds = [tokenizer.batch_decode(predictions, skip_special_tokens=<span class="literal">True</span>)] </span><br><span class="line">    </span><br><span class="line">    <span class="comment"># Replace -100 in the labels as we can&#x27;t decode them.</span></span><br><span class="line">    labels = np.where(labels != -<span class="number">100</span>, labels, tokenizer.pad_token_id)</span><br><span class="line">    decoded_labels = [tokenizer.batch_decode(labels, skip_special_tokens=<span class="literal">True</span>)] </span><br><span class="line">    <span class="keyword">return</span> metric.compute(predictions=decoded_preds, references=decoded_labels)</span><br><span class="line"></span><br><span class="line">model_name = <span class="string">&quot;ClueAI/ChatYuan-large-v1&quot;</span></span><br><span class="line">model = AutoModelForSeq2SeqLM.from_pretrained(model_name)</span><br><span class="line">tokenizer = T5Tokenizer.from_pretrained(model_name)</span><br><span class="line"></span><br><span class="line">max_input_length = <span class="number">252</span></span><br><span class="line">max_target_length = <span class="number">20</span></span><br><span class="line">batch_size = <span class="number">4</span></span><br><span class="line">prefix = <span class="string">&quot;请问下面文本属于 招聘信息、 经验贴、 求助贴 三者中的哪一类？\n&quot;</span></span><br><span class="line">suffix = <span class="string">&quot;\n选项：招聘信息, 经验贴, 求助贴\n答案：&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">preprocess_function</span>(<span class="params">examples</span>):</span><br><span class="line">    inputs = [prefix + doc + suffix <span class="keyword">for</span> doc <span class="keyword">in</span> examples[<span class="string">&quot;text&quot;</span>]]</span><br><span class="line">    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># Setup the tokenizer for targets</span></span><br><span class="line">    <span class="keyword">with</span> tokenizer.as_target_tokenizer():</span><br><span class="line">        labels = tokenizer(examples[<span class="string">&quot;target&quot;</span>], max_length=max_target_length, truncation=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">    model_inputs[<span class="string">&quot;labels&quot;</span>] = labels[<span class="string">&quot;input_ids&quot;</span>]</span><br><span class="line">    <span class="keyword">return</span> model_inputs</span><br><span class="line"></span><br><span class="line">raw_datasets = load_dataset(<span class="string">&#x27;csv&#x27;</span>, data_files=&#123;<span class="string">&#x27;train&#x27;</span>: <span class="string">&#x27;train.csv&#x27;</span>, <span class="string">&#x27;test&#x27;</span>: <span class="string">&#x27;test.csv&#x27;</span>&#125;)</span><br><span class="line">tokenized_datasets = raw_datasets.<span class="built_in">map</span>(preprocess_function, batched=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">args = Seq2SeqTrainingArguments(</span><br><span class="line">    <span class="string">f&quot;yuan-finetuned-yuan&quot;</span>,</span><br><span class="line">    evaluation_strategy = <span class="string">&quot;epoch&quot;</span>,</span><br><span class="line">    learning_rate=<span class="number">5e-5</span>,</span><br><span class="line">    per_device_train_batch_size=batch_size,</span><br><span class="line">    per_device_eval_batch_size=batch_size * <span class="number">10</span>,</span><br><span class="line">    weight_decay=<span class="number">0.01</span>,</span><br><span class="line">    save_total_limit=<span class="number">3</span>,</span><br><span class="line">    num_train_epochs=<span class="number">3</span>,</span><br><span class="line">    predict_with_generate=<span class="literal">True</span>,</span><br><span class="line">    fp16=<span class="literal">True</span></span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)</span><br><span class="line"></span><br><span class="line">trainer = Seq2SeqTrainer(</span><br><span class="line">    model,</span><br><span class="line">    args,</span><br><span class="line">    train_dataset=tokenized_datasets[<span class="string">&quot;train&quot;</span>],</span><br><span class="line">    eval_dataset=tokenized_datasets[<span class="string">&quot;test&quot;</span>],</span><br><span class="line">    data_collator=data_collator,</span><br><span class="line">    tokenizer=tokenizer,</span><br><span class="line">    compute_metrics=compute_metrics</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">trainer.train()</span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;test&quot;</span>)</span><br><span class="line"><span class="built_in">print</span>(trainer.evaluate())</span><br><span class="line">trainer.save_model(<span class="string">&quot;yuan&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h3 id="模型推理预测-1"><a href="#模型推理预测-1" class="headerlink" title="模型推理预测"></a>模型推理预测</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> transformers <span class="keyword">import</span> AutoModelForSeq2SeqLM, T5Tokenizer</span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">data = pd.read_excel(<span class="string">&quot;historical_data.xlsx&quot;</span>, sheet_name = <span class="number">0</span>).fillna(<span class="string">&quot; &quot;</span>)</span><br><span class="line">data[<span class="string">&#x27;text&#x27;</span>] = data[<span class="string">&#x27;title&#x27;</span>].apply(<span class="keyword">lambda</span> x : <span class="built_in">str</span>(x) <span class="keyword">if</span> x <span class="keyword">else</span> <span class="string">&quot;&quot;</span>) + data[<span class="string">&#x27;content&#x27;</span>].apply(<span class="keyword">lambda</span> x : <span class="built_in">str</span>(x) <span class="keyword">if</span> x <span class="keyword">else</span> <span class="string">&quot;&quot;</span>)</span><br><span class="line"></span><br><span class="line">model_name = <span class="string">&quot;yuan&quot;</span></span><br><span class="line">max_target_length = <span class="number">512</span></span><br><span class="line">model = AutoModelForSeq2SeqLM.from_pretrained(model_name)</span><br><span class="line">tokenizer = T5Tokenizer.from_pretrained(model_name)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> torch.cuda.is_available():</span><br><span class="line">    device = <span class="string">&quot;cuda:0&quot;</span></span><br><span class="line">    model.half()</span><br><span class="line"><span class="keyword">else</span>:</span><br><span class="line">    device = <span class="string">&quot;cpu&quot;</span></span><br><span class="line">model = model.to(device)</span><br><span class="line"></span><br><span class="line">prefix = <span class="string">&quot;请问下面文本属于 招聘信息、 经验贴、 求助贴 三者中的哪一类？\n&quot;</span></span><br><span class="line">suffix = <span class="string">&quot;\n选项：招聘信息, 经验贴, 求助贴\n答案：&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">get_answer</span>(<span class="params">text</span>):</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> text :</span><br><span class="line">        <span class="keyword">return</span> <span class="string">&quot;&quot;</span></span><br><span class="line">    inputs = tokenizer( prefix + <span class="built_in">str</span>(text) + suffix, return_tensors=<span class="string">&quot;pt&quot;</span>, max_length=max_target_length, truncation=<span class="literal">True</span>)</span><br><span class="line">    inputs = &#123;k:v.to(device) <span class="keyword">for</span> k,v <span class="keyword">in</span> inputs.items()&#125;</span><br><span class="line">    <span class="comment"># print(inputs)</span></span><br><span class="line">    outputs = model.generate(**inputs, max_new_tokens=<span class="number">5</span>, return_dict_in_generate=<span class="literal">True</span>)</span><br><span class="line">    <span class="keyword">return</span> tokenizer.decode(outputs[<span class="number">0</span>][<span class="number">0</span>], skip_special_tokens=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">data[<span class="string">&#x27;target&#x27;</span>] = data[<span class="string">&#x27;text&#x27;</span>].<span class="built_in">map</span>(get_answer)  <span class="comment"># not recommend, it&#x27;s better to generate in batches </span></span><br><span class="line"></span><br><span class="line">writer = pd.ExcelWriter(<span class="string">&quot;generate.xlsx&quot;</span>)</span><br><span class="line">data.to_excel(writer, index=<span class="literal">False</span>, encoding=<span class="string">&#x27;utf-8&#x27;</span>, sheet_name=<span class="string">&#x27;Sheet1&#x27;</span>)</span><br><span class="line">writer.save()</span><br><span class="line">writer.close()</span><br><span class="line"></span><br></pre></td></tr></table></figure>
      
    </div>
    <div class="article-footer">
      <blockquote class="mt-2x">
  <ul class="post-copyright list-unstyled">
    
    <li class="post-copyright-link hidden-xs">
      <strong>本文链接：</strong>
      <a href="http://chadqiu.github.io/fcb5896c5f58.html" title="如何使用huggingface的trainer训练模型？" target="_blank" rel="external">http://chadqiu.github.io/fcb5896c5f58.html</a>
    </li>
    
    <li class="post-copyright-license">
      <strong>版权声明： </strong> 本博客所有文章除特别声明外，均采用 <a href="http://creativecommons.org/licenses/by/4.0/deed.zh" target="_blank" rel="external">CC BY 4.0 CN协议</a> 许可协议。转载请注明出处！
    </li>
  </ul>
</blockquote>


<div class="panel panel-default panel-badger">
  <div class="panel-body">
    <figure class="media">
      <div class="media-left">
        <a href="https://github.com/chadqiu" target="_blank" class="img-burn thumb-sm visible-lg">
          <img src="/images/logo.jpg" class="img-rounded w-full" alt="">
        </a>
      </div>
      <div class="media-body">
        <h3 class="media-heading"><a href="https://github.com/chadqiu" target="_blank"><span class="text-dark">chadqiu</span><small class="ml-1x">Developer &amp; Researcher</small></a></h3>
        <div>在校程序猿</div>
      </div>
    </figure>
  </div>
</div>


    </div>
  </article>
  
    
  <section id="comments">
  	
      <div id="lv-container" data-id="city" data-uid="MTAyMC81Nzk4OC8zNDQ1MQ==">
        <noscript> 为正常使用来必力评论功能请激活JavaScript</noscript>
      </div>    
    
  </section>


  
</div>

  <nav class="bar bar-footer clearfix" data-stick-bottom>
  <div class="bar-inner">
  
  <ul class="pager pull-left">
    
    <li class="prev">
      <a href="/a42e1736d2ee.html" title="24-11-24Z哥直播"><i class="icon icon-angle-left" aria-hidden="true"></i><span>&nbsp;&nbsp;上一篇</span></a>
    </li>
    
    
    <li class="next">
      <a href="/6fcf23854660.html" title="如何构建一个自定义huggingface dataset数据集？"><span>下一篇&nbsp;&nbsp;</span><i class="icon icon-angle-right" aria-hidden="true"></i></a>
    </li>
    
    
    <li class="toggle-toc">
      <a class="toggle-btn " data-toggle="collapse" href="#collapseToc" aria-expanded="false" title="文章目录" role="button">    <span>[&nbsp;</span><span>文章目录</span>
        <i class="text-collapsed icon icon-anchor"></i>
        <i class="text-in icon icon-close"></i>
        <span>]</span>
      </a>
    </li>
    
  </ul>
  
  
  
  <div class="bar-right">
    
  </div>
  </div>
</nav>
  


</main>

  <footer class="footer" itemscope itemtype="http://schema.org/WPFooter">
	
	
    <ul class="social-links">
    	
        <li><a href="https://github.com/chadqiu" target="_blank" title="Github" data-toggle=tooltip data-placement=top><i class="icon icon-github"></i></a></li>
        
        <li><a href="/atom.xml" target="_blank" title="Rss" data-toggle=tooltip data-placement=top><i class="icon icon-rss"></i></a></li>
        
    </ul>

    <div class="copyright">
    	
        <div class="publishby">
        	Theme by <a href="https://github.com/cofess" target="_blank"> cofess </a>base on <a href="https://github.com/cofess/hexo-theme-pure" target="_blank">pure</a>.
        </div>
    </div>
</footer>
  <script src="//cdn.jsdelivr.net/npm/jquery@1.12.4/dist/jquery.min.js"></script>
<script>
window.jQuery || document.write('<script src="js/jquery.min.js"><\/script>')
</script>

<script src="/js/plugin.min.js"></script>


<script src="/js/application.js"></script>


    <script>
(function (window) {
    var INSIGHT_CONFIG = {
        TRANSLATION: {
            POSTS: '文章',
            PAGES: '页面',
            CATEGORIES: '分类',
            TAGS: '标签',
            UNTITLED: '(未命名)',
        },
        ROOT_URL: '/',
        CONTENT_URL: '/content.json',
    };
    window.INSIGHT_CONFIG = INSIGHT_CONFIG;
})(window);
</script>

<script src="/js/insight.js"></script>






   




   
    
<script defer type="text/javascript">
  (function(d, s) {
    var j, e = d.getElementsByTagName(s)[0];

    if (typeof LivereTower === 'function') { return; }

    j = d.createElement(s);
    j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
    j.async = true;

    e.parentNode.insertBefore(j, e);
  })(document, 'script');
</script>








</body>
</html>